#!/bin/bash
# k8s.restart.pod.sh
# check and restart pod from prometheus
# run in k8s for cronjob
# By Elvin , blog.elvin.vip update 2025-1-24

##########
#default var

#not run
k8sRun=${k8sRun:="false"}
#namespace
N=${N:=default}
#exclude
Exclude=${Exclude:="^test ^prod db$ ec-config ec-eureka"}
#wait
w=${w:=0}
#Resource usage %
Limit=${Limit:=85}
#PrometheusUrl
PUrl=${PUrl:=http://prometheus.monitor:9090}

###notice
#dingtalk token
ddtoken=${ddtoken:=""}

##########
#prometheus url
PrometheusUrl="${PUrl:=http://prometheus.monitor:9090}/api/v1/query"
URL() {
#Memory usage rate > %,
getMem="round(sum by(namespace, pod, instance, container) (container_memory_working_set_bytes{image!=\"\", namespace=\"${NameSpace}\"}) / sum by(namespace, pod, instance, container) (container_spec_memory_limit_bytes{image!=\"\", namespace=\"${NameSpace}\"}) * 100 != +Inf) > ${Limit}"
#cpu usage rate > %,
getCPU="round(sum by(namespace, pod, container) (rate(container_cpu_usage_seconds_total{image!=\"\", namespace=\"${NameSpace}\"}[10m])) / (sum by(namespace, pod, container) (container_spec_cpu_quota{image!=\"\", namespace=\"${NameSpace}\"} / 100000)) * 100) > ${Limit}"
}

##########
#pod list

podList() {
listLog="/tmp/pod.${NameSpace}.log"
>$listLog
curl -ks -m 2 "${PrometheusUrl}" --data-urlencode "query=$getMem" | jq -r ".data.result[].metric.container" >$listLog
curl -ks -m 2 "${PrometheusUrl}" --data-urlencode "query=$getCPU" | jq -r ".data.result[].metric.container" >>$listLog
sed -i '/^$/d'   $listLog
for d in $Exclude ;do sed -i "/${d}/d" $listLog; done
cp $listLog ${listLog}.txt
cat ${listLog}.txt |sort |uniq >${listLog}
}

##########
#post restart

>/tmp/pod.r.sh
for NameSpace in $N; do
URL
podList
if [ $(cat /tmp/pod.${NameSpace}.log |wc -l) -ge 1 ];then
echo "#${NameSpace}" >>/tmp/pod.r.sh
w=${w:=0}
if [ $w -ne 0 ];then
    c=$w
elif [ $(cat $listLog |wc -l) -le 5 ];then
    c=15
elif [ $(cat $listLog |wc -l) -le 10 ];then
    c=20
elif [ $(cat $listLog |wc -l) -le 15 ];then
    c=30
else
    c=40
fi
cat $listLog |sed "s/.*/kubectl rollout restart deploy -n ${NameSpace} & ; sleep ${c} /g" >>/tmp/pod.r.sh
echo "" >>/tmp/pod.r.sh
fi
done
cat /tmp/pod.r.sh

if [ "${k8sRun}" = "false" ];then
    echo "#do not run"
else
    echo "#pod restarting"
    bash /tmp/pod.r.sh
fi

##########
#dd notice

if [ "$ddtoken" != "" -a "${k8sRun}" != "false" ];then
MsgTxt() {
add_title="notice from restart"
echo "_notice from k8s_ __restart__ | _usage > ${Limit}%_    ">/tmp/msg.k8s.log
for n in $N; do
NameSpace=$n
if [ $(cat /tmp/pod.${NameSpace}.log |wc -l) -ge 1 ];then
    awk '{print NR". '$n'/"$NF"    "}' /tmp/pod.${NameSpace}.log >>/tmp/msg.k8s.log
fi
done
[ $(cat /tmp/msg.k8s.log |wc -l) -eq 1 ] && { echo "nothing"; exit; }
export ddtxt="$(cat /tmp/msg.k8s.log)"
}

#send
[ $(grep "rollout" /tmp/pod.r.sh |wc -l) -eq 0 ] && { echo ""; echo "#nothing notice"; exit ; }
MsgTxt
# export ddtoken=${ddtoken}
export Title="k8s pod restart"
curl -s  http://files.elvin.vip/shell/ddmsg.url.md.sh |bash
fi
##########
#https://gitee.com/alivv/elvin-demo/raw/master/shell/k8s.restart.pod.sh
